from bs4 import BeautifulSoup
from modules.web_scraper import WebPageScraper
from tqdm import tqdm

Urls = []
with open('./data/urls.txt', 'r', encoding='utf-8') as f:
    for line in f.readlines():    
        Urls.append(line.strip())

scraper = WebPageScraper('https://bbs.mihoyo.com/ys/obc/channel/map/80/212?bbs_presentation_style=no_header')
links = scraper.get_a_tags_href()


# 初始化进度条，指定总进度
progress_bar = tqdm(total=len(links))

# 创建一个空列表来存储视频源
videosrc_list = []

# 迭代链接列表
for link in links:
    # 拼接完整的链接
    link = 'https://bbs.mihoyo.com' + link
    # 抓取网页内容
    videoPage = WebPageScraper(link)
    # 找到视频源
    videosrc = videoPage.find_first_video_src()
    # 将视频源添加到列表中
    videosrc_list.append(videosrc)
    # 更新进度条
    progress_bar.update(1)

# 完成后关闭进度条
progress_bar.close()

# 打印总链接数量
print(f"总链接数量：{len(links)}")
